# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
import re
import datetime
# py2.7和py3（parse）的urlpasre所在的包不一样
import urlparse
import sys
from ArticleSpider.items import JobBoLeArticleItem, ArticleItemLoader
from ArticleSpider.utils.common import get_md5
from scrapy.loader import ItemLoader
reload(sys)
sys.setdefaultencoding('utf-8')


class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/', ]
    # start_urls = ['http://blog.jobbole.com/112218/']
    # 找到想要查看的变量，点击右键，可以看到：Add to Watches， View as array

    def parse(self, response):
        """
        1.获取文章列表页url并交给scrapy下载并进行解析
        2.获取写一页的url并交给scrapy进行下载，下载完成后交给parse函数
        """
        # title = response.xpath("//div[@class='entry-header']/h1/text()").extract()[0]
        # create_date = response.xpath("//p[@class='entry-meta-hide-on-mobile']").extract()[0].strip().replace("·", "")
        # praise_nums = response.xpath("//span[contains(@class,'vote-post-up')]/text()").extract()[0]
        # fav_nums = response.xpath("//span[contains(@class,'bookmark-btn')]/text()").extract()[0]
        # match_re = re.match(".*(\d+).*", fav_nums)
        # if match_re:
        #     fav_nums = match_re.group(1)
        #
        # comment_nums = response.xpath("//a[@href='#article-comment']/span").extract()[0]
        # match_re = re.match(".*(\d).*", comment_nums)
        # if match_re:
        #     comment_nums = match_re.group(1)
        #
        # content = response.xpath("//div[@class='entry']").extract()[0].replace(u"\ax0", u'')
        # tag_list =  response.css("div.entry").extract()[0].replace(u"\xa0", '0')
        # tag_list = [ele for ele in tag_list if not ele.endwith(u"评论")]
        # css选择器限定的需要获取的文章链接
        post_nodes = response.css("div#archive div.floated-thumb div.post-thumb a")
        for post_node in post_nodes:
            image_url = post_node.css("img::attr(src)").extract_first("")
            # 拼接url  提交给 scrapy
            post_url = post_node.css("::attr(href)").extract_first("")
            yield Request(url=urlparse.urljoin(response.url, post_url), meta={"front_image_url":image_url, }, callback=self.parse_detail)
        # 提取下一页并交给scrapy进行下载    测试同时拥有两个属性的class  ：next 和 page-numbers
        #
        next_url = response.css("div.navigation a.next.page-numbers::attr(href)").extract_first()

        if next_url:
            yield Request(url=urlparse.urljoin(response.url, next_url), callback=self.parse)

    #  提取文章的字段 #css选择器提取字段

    def parse_detail(self, response):

        # article_item = JobBoLeArticleItem()
        # title = response.css("div.entry-header h1::text").extract_first("")
        # create_time = response.css("p.entry-meta-hide-on-mobile::text").extract_first("").strip().replace(u"\xb7", "")
        # praise_nums = response.css(".vote-post-up h10::text").extract_first("").strip()
        # front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
        # fav_nums = response.css("span.bookmark-btn h10::text").extract_first("")
        # match_re = re.match(".*(\d+).*", fav_nums)
        # if match_re:
        #     fav_nums = int(match_re.group(1))
        # else:
        #     fav_nums = 0
        # comment_nums = response.css("a[href='#article-comment'] span::text").extract_first("")
        # match_re = re.match(".*(\d+).*", comment_nums)
        # if match_re:
        #     comment_nums = int(match_re.group(1))
        # else:
        #     comment_nums = 0
        # col_nums = response.css("span.bookmark-btn::text").extract_first("")
        # match_re = re.match(".*(\d+).*", col_nums)
        # if match_re:
        #     col_nums = int(match_re.group(1))
        # else:
        #     col_nums = 0
        # #    response.css("div.entry::text").extract()[0].replace(u"\xa0", '0')  加入text函数，会打印空，
        # # 因为text是打印文本，而忽略子标签的内容，不论子标签是否有文字
        # content = response.css("div.entry").extract()[0].replace(u"\xa0", '0')
        #
        # tag_list = response.css("p.entry-meta-hide-on-mobile a").extract()
        # # tag_list = [ele for ele in tag_list if not str(ele).endwith("评论")]
        # tags = ','.join(tag_list)
        # article_item["url_object_id"] = get_md5(response.url)
        # article_item["title"] = title
        # try:
        #     create_time = datetime.datetime.strptime(create_time, "%Y%m%d").date()
        # except Exception as e:
        #     create_time = datetime.datetime.now().date()
        #
        # article_item["create_time"] = create_time
        # article_item["url"] = response.url
        # # 默认的是当成列表来处理
        # article_item["front_image_url"] = [front_image_url]
        # article_item["praise_nums"] = praise_nums
        # article_item["comment_nums"] = comment_nums
        # article_item["fav_nums"] = fav_nums
        # article_item["tags"] = tags
        # article_item["content"] = content

        # 通过ItemLoader加载item

        front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
        item_loader = ArticleItemLoader(item=JobBoLeArticleItem(), response=response)
        item_loader.add_css("title", "div.entry-header h1::text")
        item_loader.add_xpath("praise_nums", "//span[contains(@class,'vote-post-up')]/text()")
        item_loader.add_css("comment_nums", "a[href='#article-comment'] span::text")
        item_loader.add_css("fav_nums", "span.bookmark-btn::text")
        item_loader.add_css("create_time", "p.entry-meta-hide-on-mobile::text")
        item_loader.add_css("content", "div.entry")
        item_loader.add_css("tags", "p.entry-meta-hide-on-mobile a")
        item_loader.add_value("url", response.url)
        item_loader.add_value("front_image_url", [front_image_url])
        item_loader.add_value("url_object_id", get_md5(response.url))

        article_item = item_loader.load_item()
        yield article_item

        pass
